/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package com.feheadline.ml.classification;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.ToAnalysis;

import com.aliasi.tokenizer.Tokenizer;

/**
 *
 * @author Administrator
 */
public class AnsjTokenizer extends Tokenizer{
    private static final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(AnsjTokenizer.class); 
    private List<Term> parse = new ArrayList<Term>();
    private int currentPos = -1;
    private HashSet<String> ignoreNatrue = new HashSet<String>(){{
            add("null");//null
            add("p");//介词
            add("w");//标点
            add("u");//助词
            add("e");//叹词
            add("o");//拟声词
        }};
    
    public AnsjTokenizer(char[] ch, int offset, int length) {
        String text = String.valueOf(ch);//new String(String.valueOf(ch).getBytes("gbk"),"utf-8");
        List<Term> terms = ToAnalysis.parse(text);
        for(Term term:terms){
            String natureStr = term.getNatrue().natureStr;
            if(natureStr!=null && !ignoreNatrue.contains(natureStr))parse.add(term);
        }
    }

    @Override
    public String nextToken() {
        if(parse==null||currentPos>=parse.size()-1)return null;
        else {
            currentPos++;
            Term term = parse.get(currentPos);
            return term.getName()+"/"+term.getNatrue().natureStr;
        }
    }
    
}
